from jili.core.printlog import print
import time,datetime,sys
from jili.calc.combine import allpairs
from jili.data.db import get_calender,getdb_client,tradecal
from copy import deepcopy
from jili.dataset import dataset
from jili.dataset import factordata
def getkey(i,keys):
    key="k"
    for k in keys:
        if k in i.keys():
            key=key+"-"+str(i[k])
    return key
def check_cu_index(cu,keys):
    indexs0=[]
    for i in cu.list_indexes():
        v = i.to_dict()
        if v["name"] != "_id_":
            k=list(v["key"])
            indexs0.append(k)
    for key in keys:
        if isinstance(key,str):
            if [key] not in indexs0:
                cu.create_index([(key,1)], name=key+"1", unique=False)
                print("create_index",key+"1")
                indexs0.append([key])
        elif isinstance(key,dict):
            if list(key.keys()) not in indexs0:
                name=""
                indexkey=[]
                for kki,vvi in key.items():
                    name=name+"_"+kki
                    indexkey.append((kki,vvi))
                cu.create_index(indexkey, name=name + "_1", unique=False)
                print("create_index", indexkey)
                indexs0.append([key])
        else:
            if list(key) not in indexs0:
                name = ""
                indexkey = []
                for kki in key:
                    name = name + "_" + kki
                    indexkey.append((kki, 1))
                cu.create_index(indexkey, name=name + "_1", unique=False)
                print("create_index", indexkey)
                indexs0.append([key])

class run_frame:
    def __init__(self,config):
        self.config=config
        self.isdebug =True
        stop_arg=config.get("stop_arg",None)
        self.set_stop_arg(stop_arg)
        self.cu=None
        self.count=0
        if "mongo_args" in config.keys():
            cu=self.get_cu()
            keys=config["mongo_args"]["keys"]
            check_cu_index(cu,keys)
    def get_cu(self):
        if self.cu is not None:
            return self.cu
        mongo_args=self.config["mongo_args"]
        if "mongodb_cu" in mongo_args.keys():
            self.cu=mongo_args["mongodb_cu"]
        else:
            db=getdb_client(dbname=mongo_args["mongo_db"],ip=mongo_args["mongo_ip"],port=mongo_args["mongo_port"],auth=mongo_args["mongo_auth"])
            self.cu = db[mongo_args["mongo_cu"]]
        return self.cu
    def set_stop_arg(self,stop_arg):
        self.stop_arg = stop_arg
    def mongo_save(self,r,config,arg,deal):
        if r:
            cu=self.get_cu()
            have=self.config["mongo_args"].get("mongo_have",None)
            isupdate=deal.get("isupdate",True)
            if have is not None:
                keys = self.config["mongo_args"]["keys"]
                n0=0
                n1=0
                for i in r:
                    key=getkey(i,keys)
                    if key in have.keys():
                        n0=n0+1
                        if isupdate:
                            cu.update_one({"_id":have[key]}, {"$set": i})
                        continue
                    n1 = n1 + 1
                    cu.insert_one(i)
                print("mongo_save", len(r),n0,n1,config["call"].__name__, self.__args2str(arg))
            else:
                try:
                    cu.insert_many(r)
                except Exception as e:
                    print(e,r[0])
                    for i in r:
                        cu.insert_one(i)
                print("mongo_save",len(r),config["call"].__name__,self.__args2str(arg))
        return r
    def data_add_field(self,r,config,arg,deal):
        add={}
        for k,v in deal.items():
            if k=="arg":
                for vv in v:
                    if isinstance(vv,str):
                        add[vv]=arg[vv]
                    else:
                        add[vv[0]] = arg[vv[1]]
            else:
                for vv in v:
                    add[vv] = config[vv]
        for t in r:
            t.update(add)
        return r
    def data_alter_field(self,r,config,arg,deal):
        for t in r:
            for i in deal:
                i1 = i[0]
                i2 = i[1]
                if len(i) == 3:
                    t[i1] = i[2](t)
                else:
                    t[i1] = t[i2]
        return r
    def data_del_field(self,r,config,arg,deal):
        for t in r:
            for i in deal:
                if i in t.keys():
                    del t[i]
        return r
    def deals_check_mongo_args(self,batch_args,check_mongo_args):
        if check_mongo_args is not None:
            sort = check_mongo_args["sort"]  # [("timekey",-1)]
            keys = check_mongo_args.get("keys",[])  # ["timekey"]/[("timekey","tradedate")]
            if keys==[]:
                for i in sort:
                    keys.append(i[0])
            find_arg=check_mongo_args.get("find_arg",{})
            ischeck=check_mongo_args.get("ischeck",True)
            back_days=check_mongo_args.get("back_days",0)
            if ischeck:
                cu=self.get_cu()
                index={}
                for i in cu.find(find_arg).sort(sort):
                    for ks in keys:
                        if isinstance(ks,str):
                            k=ks
                            if k in i.keys():
                                index[k] = i[k]
                        else:
                            k=ks[0]
                            k1=ks[1]
                            if k in i.keys():
                                index[k1] = i[k]
                    break
                if index=={}:
                    return batch_args
                batch_args = deepcopy(batch_args)
                sys_genarg=self.config.get("sys_genarg",None)
                if sys_genarg is not None:
                    region = sys_genarg.get("region", "C")
                    if sort[0][1]==-1:#最大  倒叙
                        if back_days==0:
                            sys_genarg["start"] = tradecal.getnexttradeday(index[sort[0][0]],region=region)
                        elif back_days==1:
                            find = deepcopy(find_arg)
                            find.update(index)
                            sys_genarg["start"] = index[sort[0][0]]
                        else:
                            d0=tradecal.get_diff_tradeday(index[sort[0][0]],diff=-back_days+1,region=region)
                            find = deepcopy(find_arg)
                            find.update(index)
                            find[sort[0][0]] = {"$gte": d0, "$lte": index[sort[0][0]]}
                            sys_genarg["start"]=d0
                    else:
                        if back_days==0:
                            sys_genarg["end"] = tradecal.getpretradeday(index[sort[0][0]],region=region)
                        elif back_days==1:
                            find = deepcopy(find_arg)
                            find.update(index)
                            sys_genarg["end"] = index[sort[0][0]]
                        else:
                            d0=tradecal.get_diff_tradeday(index[sort[0][0]],diff=back_days-1,region=region)
                            find = deepcopy(find_arg)
                            find.update(index)
                            find[sort[0][0]]={"$gte":index[sort[0][0]],"$lte":d0}
                            sys_genarg["end"]=d0
                    if back_days > 0:
                        keyss=self.config["mongo_args"]["keys"]
                        have={}
                        for i in cu.find(find):
                            key=getkey(i,keyss)
                            have[key]=i["_id"]
                        self.config["mongo_args"]["mongo_have"]=have
                else:
                    for k,v in index.items():
                        n=batch_args[k].index(v)
                        m=n-back_days+1
                        batch_args[k]=batch_args[k][m:]
        return batch_args

    def deals_sys_genarg(self,batch_argss,sys_genarg):
        batch_args=deepcopy(batch_argss)
        if sys_genarg is not None:
            mode=sys_genarg["mode"]
            if mode=="tradeday" or mode=="tradeday_zone":
                start = sys_genarg["start"]
                end = sys_genarg.get("end", None)
                keys = sys_genarg["keys"]
                isreverse = sys_genarg.get("isreverse", False)
                region = sys_genarg.get("region", "C")
                freq = sys_genarg.get("freq", None)
                calender = get_calender(start, end, region=region)
                if isinstance(keys,str):
                    i=keys
                    if i in batch_args.keys():
                        del batch_args[i]
                else:
                    for i in keys:
                        if i in batch_args.keys():
                            del batch_args[i]
                calender.sort(reverse=isreverse)
                # if isreverse:
                #     calender.reverse()
                if freq is not None:
                    calender0=[]
                    for i in calender:
                        calender0.append(i.strftime(freq))
                    calender=calender0
                if mode=="tradeday":
                    if calender:
                        batch_args[keys]=calender
                    else:
                        batch_args=[]
                else:
                    zone_days=sys_genarg["zone_days"]
                    zones=[]
                    for i in range(0,len(calender),zone_days):
                        zones.append(calender[i:i+zone_days])
                    if batch_args:
                        # allpairs_args,一级参数batch_args时有效{}#allpairs的参数
                        allpairs_args = self.config.get("allpairs_args",
                                                        {})  # allpairs的参数 filter_func=None,filter_funcbylist=None,keys=None,add_data={}
                        allpairs_args["argss"] = batch_args
                        args_list0 = list(allpairs(**allpairs_args))
                        args_list = []
                        for arg0 in args_list0:
                            for i in zones:
                                arg=deepcopy(arg0)
                                arg[keys[0]]=i[0]
                                arg[keys[1]] = i[-1]
                                args_list.append(arg)
                    else:
                        args_list = []
                        for i in zones:
                            args_list.append({keys[0]:i[0],keys[1]:i[-1]})
                    return args_list
            elif mode=="key":
                keys=sys_genarg["keys"]
                # keylist=sys_genarg["keylist"]
                key_find=sys_genarg.get("key_find", {})
                islist=sys_genarg.get("islist",False)
                cu = self.get_cu()
                if isinstance(keys,str):
                    save_keys=cu.distinct(keys,key_find)
                    if not islist:
                        now_keys = set(batch_args[keys]) - set(save_keys)
                        if now_keys:
                            batch_args[keys] = list(now_keys)
                        else:
                            batch_args=[]
                    else:
                        now_keys = set(batch_args[keys][0]) - set(save_keys)
                        if now_keys:
                            batch_args[keys]=[list(now_keys)]
                        else:
                            batch_args=[]
                elif isinstance(keys,tuple):
                    save_keys = cu.distinct(keys[0], key_find)

                    if not islist:
                        now_keys = set(batch_args[keys[1]]) - set(save_keys)
                        if now_keys:
                            batch_args[keys[1]] = list(now_keys)
                        else:
                            batch_args=[]
                    else:
                        now_keys = set(batch_args[keys[1]][0]) - set(save_keys)
                        if now_keys:
                            batch_args[keys[1]] = [list(now_keys)]
                        else:
                            batch_args=[]
                else:
                    save_keys={}
                    for i in cu.find(key_find):
                        for k in keys:
                            if isinstance(k,str):
                                k0=k
                                k1=k
                            else:
                                k0=k[0]
                                k1=k[1]
                            if k0 in i.keys():
                                if k1 not in save_keys.keys():
                                    save_keys[k1]=[]
                                save_keys[k1].append(i[k0])
                    now_keys={}
                    if not islist:
                        for k, v in save_keys.items():
                            now_keys[k] = set(batch_args[k]) - set(v)
                        for k,v in now_keys.items():
                            batch_args[k]=list(v)
                    else:
                        for k, v in save_keys.items():
                            now0=set(batch_args[k][0]) - set(v)
                            if now0:
                                now_keys[k] = now0
                        if now_keys:
                            for k,v in now_keys.items():
                                batch_args[k]=[list(v)]
                        else:
                            batch_args=[]
        return batch_args
    def deals(self,r,arg):
        deals0=self.config.get("data_etls",[])
        if isinstance(deals0,dict):  #dict
            for k,deal in deals0.items():
                if isinstance(deal, list) or isinstance(deal, tuple):
                    deal1 = deal[0]
                    arg1 = deal[1]
                else:
                    deal1 = deal
                    arg1 = {}
                if isinstance(deal1, str):
                    deal0 = getattr(self, deal1)
                    r = deal0(r, self.config, arg, arg1)
                else:
                    r = deal1(r, self.config, arg, arg1)
        else:#list
            for deal in deals0:
                if isinstance(deal,list) or isinstance(deal,tuple):
                    deal1=deal[0]
                    arg1=deal[1]
                else:
                    deal1 = deal
                    arg1 = {}
                if isinstance(deal1,str):
                    deal0=getattr(self,deal1)
                    r = deal0(r, self.config, arg, arg1)
                else:
                    r = deal1(r, self.config, arg, arg1)
        self.count=self.count+len(r)
    def check_stop(self):
        if self.stop_arg is not None:
            if isinstance(self.stop_arg,list):
                stop_args=self.stop_arg
            else:
                stop_args=[self.stop_arg]
            for stop_arg in stop_args:
                call0,call_arg,deal_flag=stop_arg
                if isinstance(call0,str):
                    call00=getattr(self,call0)
                else:
                    call00=call0
                r = call00(**call_arg)
                if r:
                    if deal_flag =="hold":
                        while r:
                            print("check_stop hold sleep 60",self.stop_arg)
                            time.sleep(60)
                            r = call00(**call_arg)
                    else:
                        print("check_stop exit",self.stop_arg)
                        sys.exit(0)
    def stop_times(self,stop_times):
        isclose = False
        if stop_times is not None:
            now = datetime.datetime.now()
            if isinstance(stop_times,list):
                if isinstance(stop_times[0], str):
                    now = now.strftime("%H:%M")
                if now >= stop_times[0] and now <= stop_times[1]:
                    isclose = True
            else:
                if isinstance(stop_times,str):
                    now=now.strftime("%H:%M")
                if now >= stop_times:
                    isclose = True
        return isclose
    def print_arg(self,arg):
        arg0={}
        for k,v in arg.items():
            arg0[k] = v
            if isinstance(v,list):
                if len(v)>5:
                    arg0[k]=v[:5]
                    arg0[k].append("other"+str(len(v)-5))
        return arg0
    def __args2str(self,arg,isdict=False):
        if isdict:
            s = {}
            for k, v in arg.items():
                if isinstance(v, list):
                    if len(v) > 5:
                        s[k]=str(v[:5]) + " other" + str(len(v) - 5)
                    else:
                        s[k]=v
                else:
                    s[k]=v
        else:
            s=""
            for k,v in arg.items():
                if isinstance(v,list):
                    if len(v)>5:
                        s=s+k+":"+str(v[:5])+"other"+str(len(v)-5)+" "
                    else:
                        s=s+k+":"+str(v)+" "
                else:
                    s=s+" "+k+"="+str(v)
        return s
    def run_call(self,arg):
        run_sleep = self.config.get("run_sleep", None)
        call = self.config["call"]
        if "arg_limit" in self.config.keys():
            arg_limit=self.config["arg_limit"]
            arg[arg_limit[0]]=arg_limit[1]
            offset=arg_limit[2]
            arg[offset]=0
            if self.isdebug:
                print("call(**arg)",call,self.__args2str(arg,True))
            else:
                r = call(**arg)
                if r is not None:
                    print("call", self.config["call"].__name__,len(r), self.print_arg(arg))
                    if len(arg_limit)>3:
                        n=arg_limit[3](r)
                    else:
                        n=len(r)
                    self.deals(r,arg)
                    while n==arg_limit[1]:
                        if run_sleep is not None:
                            time.sleep(run_sleep)
                        arg[offset] = arg[offset]+n
                        self.check_stop()
                        r = call(**arg)
                        if r is not None:
                            print("call", self.config["call"].__name__, len(r), self.print_arg(arg))
                            if len(arg_limit) > 3:
                                n = arg_limit[3](r)
                            else:
                                n = len(r)
                            self.deals(r,arg)
                        else:
                            print("call", self.config["call"].__name__, "no data", self.print_arg(arg))
                else:
                    print("call", self.config["call"].__name__, "no data", self.print_arg(arg))
        else:
            if self.isdebug:
                print("call(**arg)",call,self.__args2str(arg,True))
            else:
                self.check_stop()
                r = call(**arg)
                if r is not None:
                    print("call",self.config["call"].__name__,len(r), self.print_arg(arg))
                    self.deals(r,arg)
                else:
                    print("call", self.config["call"].__name__, "no data", self.print_arg(arg))
    def run_call_bybatch(self,args_list):
        run_sleep=self.config.get("run_sleep",None)
        for arg in args_list:
            self.run_call(arg)
            if run_sleep is not None:
                time.sleep(run_sleep)
    def run(self,isdebug=True,config=None):
        if config is not None:
            self.config=config
        self.isdebug=isdebug
        #优先级:arg>args_list>batch_args
        if "arg" in self.config.keys():
            arg=self.config["arg"]
            self.run_call(arg)
            msg = self.__args2str(arg)
        elif "args_list" in self.config.keys():
            args_list=self.config["args_list"]
            self.run_call_bybatch(args_list)
            msg = self.__args2str(args_list[0])+"~"+self.__args2str(args_list[-1])
        elif "batch_args" in self.config.keys():
            batch_args=self.config["batch_args"]
            #check_mongo_args:检查当前数据,一级参数 batch_args有效
            check_mongo_args=self.config.get("check_mongo_args",None)
            batch_args = self.deals_check_mongo_args(batch_args, check_mongo_args)
            #sys_genarg:一级参数batch_args时有效,系统参数生成模式:支持按日期,按时间周期的定义
            sys_genarg=self.config.get("sys_genarg",None)
            batch_args=self.deals_sys_genarg(batch_args,sys_genarg)
            if isinstance(batch_args,dict):
                #allpairs_args,一级参数batch_args时有效{}#allpairs的参数
                allpairs_args=self.config.get("allpairs_args",{}) #allpairs的参数 filter_func=None,filter_funcbylist=None,keys=None,add_data={}
                allpairs_args["argss"]=batch_args
                args_list=allpairs(**allpairs_args)
            else:
                args_list=batch_args
            self.run_call_bybatch(args_list)
            if not isinstance(args_list,list):
                args_list=list(args_list)
            if len(args_list)>1:
                msg = self.__args2str(args_list[0]) + "~" + self.__args2str(args_list[-1])
            elif len(args_list)==1:
                msg = self.__args2str(args_list[0])
            else:
                msg=""
        if "isreport_fs_sheet" in self.config.keys():
            isreport_fs_sheet=self.config["isreport_fs_sheet"]
            d=datetime.datetime.now()
            end=datetime.datetime(d.year,d.month,d.day)
            t = {"数据": isreport_fs_sheet, "日期": int(end.timestamp() * 1000), "数量": self.count, "是否正常": "正常", "备注": msg}
            from mohe.ibot.report_fs_sheet import add_records
            add_records(t)


    def dataset_save(self,r,config,arg,deal):
        tool=dataset(**deal)
        tool.save(r)
        return r
    def factor_save(self,r,config,arg,deal):
        tool = factordata(**deal)
        tool.save_all(r)
        return r
class data_run:
    def __init__(self):
        self.ss={}
        self.n=0
        self.ispass=True
        self.stop_arg=None
    def add(self,ss,name=None):
        if name is None:
            self.n=self.n+1
            name="data_run"+str(self.n)
        if isinstance(ss,dict):
            self.ss[name]=run_frame(ss)
        else:
            self.ss[name]=ss
    def run(self,ispass=True,stop_arg=None,isdebug=True):
        if stop_arg is not None:
            self.stop_arg=stop_arg
        if self.stop_arg is not None:
            for k, v in self.ss.items():
                v.set_stop_arg(stop_arg)
        self.ispass=ispass
        if self.ispass:
            for k,v in self.ss.items():
                try:
                    v.run(isdebug=isdebug)
                    print(k,"run over")
                except Exception as e:
                    print(k,"run fail",e)
        else:
            for k, v in self.ss.items():
                v.run(isdebug=isdebug)
                print(k, "run over")


